﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using HtmlAgilityPack;

namespace WebCrawler
{
    class Program
    {
        static readonly HttpClient client = new HttpClient();
        static readonly Regex phoneRegex = new Regex(@"\b\d{8}\b");
        static readonly int targetPhoneNumberCount = 100;

        static async Task Main(string[] args)
        {
            var keyword = Console.ReadLine(); 

            var urls = await GetSearchUrls(keyword);

            var uniquePhoneNumbers = new ConcurrentDictionary<string, List<string>>();
            var visitedUrls = new ConcurrentBag<string>();

            var tasks = urls.Select(url => Task.Run(async () =>
            {
                visitedUrls.Add(url);
                var phoneNumbers = await ExtractPhoneNumbersFromUrl(url);
                foreach (var phoneNumber in phoneNumbers)
                {
                    uniquePhoneNumbers.AddOrUpdate(phoneNumber, new List<string> { url }, (key, value) => { value.Add(url); return value; });
                }
            })).ToList();

            while (uniquePhoneNumbers.Count < targetPhoneNumberCount && tasks.Any())
            {
                var finishedTask = await Task.WhenAny(tasks);
                tasks.Remove(finishedTask);
            }

            foreach (var pair in uniquePhoneNumbers)
            {
                Console.WriteLine($"电话号码: {pair.Key}, URLs: {string.Join(", ", pair.Value)}");
            }

            Console.WriteLine($"已爬取的URL: {string.Join(", ", visitedUrls)}");

            Console.WriteLine("按任意键退出...");
            Console.ReadLine(); 
        }

        static async Task<List<string>> GetSearchUrls(string keyword)
        {
            var tasks = new List<Task<List<string>>>()
            {
                GetSearchUrlsFromGoogle(keyword),
                GetSearchUrlsFromBing(keyword),
                GetSearchUrlsFromBaidu(keyword)
            };

            var results = await Task.WhenAll(tasks);

            return results.SelectMany(x => x).Distinct().ToList();
        }

        static async Task<List<string>> GetSearchUrlsFromGoogle(string keyword)
        {
            var htmlDocument = new HtmlDocument();
            var searchUrl = "https://www.google.com/search?q=" + Uri.EscapeDataString(keyword);
            var response = await client.GetAsync(searchUrl);
            var htmlContent = await response.Content.ReadAsStringAsync();
            htmlDocument.LoadHtml(htmlContent);

            var searchResultNodes = htmlDocument.DocumentNode.SelectNodes("//div[@class='g']//a");
            if (searchResultNodes == null || searchResultNodes.Count == 0) return new List<string>();

            var urls = new List<string>();
            foreach (var node in searchResultNodes)
            {
                var link = node.GetAttributeValue("href", string.Empty);
                if (!string.IsNullOrEmpty(link) && link.StartsWith("http"))
                {
                    urls.Add(link);
                }
            }

            return urls;
        }

        static async Task<List<string>> GetSearchUrlsFromBing(string keyword)
        {
            var htmlDocument = new HtmlDocument();
            var searchUrl = "https://www.bing.com/search?q=" + Uri.EscapeDataString(keyword);
            var response = await client.GetAsync(searchUrl);
            var htmlContent = await response.Content.ReadAsStringAsync();
            htmlDocument.LoadHtml(htmlContent);

            var searchResultNodes = htmlDocument.DocumentNode.SelectNodes("//li[@class='b_algo']//h2/a");
            if (searchResultNodes == null || searchResultNodes.Count == 0) return new List<string>();

            var urls = new List<string>();
            foreach (var node in searchResultNodes)
            {
                var link = node.GetAttributeValue("href", string.Empty);
                if (!string.IsNullOrEmpty(link) && link.StartsWith("http"))
                {
                    urls.Add(link);
                }
            }

            return urls;
        }
        static async Task<List<string>> GetSearchUrlsFromBaidu(string keyword)
        {
            var htmlDocument = new HtmlDocument();
            var searchUrl = "https://www.baidu.com/s?wd=" + Uri.EscapeDataString(keyword);
            var response = await client.GetAsync(searchUrl);
            var htmlContent = await response.Content.ReadAsStringAsync();
            htmlDocument.LoadHtml(htmlContent);

            var searchResultNodes = htmlDocument.DocumentNode.SelectNodes("//div[@class='result c-container ']/h3/a");
            if (searchResultNodes == null || searchResultNodes.Count == 0) return new List<string>();

            var urls = new List<string>();
            foreach (var node in searchResultNodes)
            {
                var link = node.GetAttributeValue("href", string.Empty);
                if (!string.IsNullOrEmpty(link))
                {
                    urls.Add(link);
                }
            }

            return urls;
        }

        static async Task<List<string>> ExtractPhoneNumbersFromUrl(string url)
        {
            var htmlDocument = new HtmlDocument();
            try
            {
                var response = await client.GetAsync(url);
                var htmlContent = await response.Content.ReadAsStringAsync();
                htmlDocument.LoadHtml(htmlContent);
            }
            catch
            {
                return new List<string>();
            }

            var phoneNumbers = new List<string>();
            var matches = phoneRegex.Matches(htmlDocument.Text);
            foreach (Match match in matches)
            {
                if (match.Success)
                {
                    phoneNumbers.Add(match.Value);
                }
            }

            return phoneNumbers;
        }
    }
}

